j62
/
ctbrec
2
2
Fork 2
Code Issues 5 Pull Requests Packages Projects Releases 3 Wiki Activity

Generate / update playlist while recording

This commit is contained in:
0xb00bface 2021-09-10 21:37:11 +02:00
parent 9c2a8242de
commit b959c57b8f
16 changed files with 197 additions and 367 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
client/src/main/java/ctbrec/RecordingDownload.java
View File

@ -17,6 +17,7 @@ import ctbrec.io.HttpException;
import ctbrec.recorder.ProgressListener; import ctbrec.recorder.ProgressListener;
import ctbrec.recorder.download.hls.MergedFfmpegHlsDownload; import ctbrec.recorder.download.hls.MergedFfmpegHlsDownload;
import ctbrec.recorder.download.hls.SegmentPlaylist; import ctbrec.recorder.download.hls.SegmentPlaylist;
import ctbrec.recorder.download.hls.SegmentPlaylist.Segment;
import okhttp3.Request; import okhttp3.Request;
import okhttp3.Response; import okhttp3.Response;
@ -56,8 +57,8 @@ public class RecordingDownload extends MergedFfmpegHlsDownload {
SegmentPlaylist segmentPlaylist = getNextSegments(segmentPlaylistUri); SegmentPlaylist segmentPlaylist = getNextSegments(segmentPlaylistUri);
long loadedBytes = 0; long loadedBytes = 0;
for (String segmentUrl : segmentPlaylist.segments) { for (Segment segment : segmentPlaylist.segments) {
loadedBytes += downloadFile(segmentUrl, loadedBytes, sizeInBytes, progressListener); loadedBytes += downloadFile(segment.url, loadedBytes, sizeInBytes, progressListener);
int progress = (int) (loadedBytes / (double) sizeInBytes * 100); int progress = (int) (loadedBytes / (double) sizeInBytes * 100);
progressListener.update(progress); progressListener.update(progress);
} }

4
client/src/main/resources/html/docs/ConfigurationFile.md
View File

@ -24,10 +24,6 @@ until a recording is finished. 0 means unlimited.
- **determineResolution** (app only) - [`true`,`false`] Display the stream resolution on the thumbnails. - **determineResolution** (app only) - [`true`,`false`] Display the stream resolution on the thumbnails.
- **fastPlaylistGenerator** (server only) - Use a fast playlist generator, which is not as accurate. This might lead to inaccurate skipping and timelines in media players. Useful for weak devices.
- **generatePlaylist** (server only) - [`true`,`false`] Generate a playlist once a recording terminates.
- **hlsdlExecutable** - Path to the hlsdl executable, which is used, if `useHlsdl` is set to true - **hlsdlExecutable** - Path to the hlsdl executable, which is used, if `useHlsdl` is set to true
- **httpPort** - [1 - 65536] The TCP port, the server listens on. In the server config, this will tell the server, which port to use. In the application this will set - **httpPort** - [1 - 65536] The TCP port, the server listens on. In the server config, this will tell the server, which port to use. In the application this will set

2
common/src/main/java/ctbrec/Settings.java
View File

@ -69,7 +69,6 @@ public class Settings {
public List<String> disabledSites = new ArrayList<>(); public List<String> disabledSites = new ArrayList<>();
public String downloadFilename = "${modelSanitizedName}-${localDateTime}"; public String downloadFilename = "${modelSanitizedName}-${localDateTime}";
public List<EventHandlerConfiguration> eventHandlers = new ArrayList<>(); public List<EventHandlerConfiguration> eventHandlers = new ArrayList<>();
public boolean fastPlaylistGenerator = false;
public boolean fastScrollSpeed = true; public boolean fastScrollSpeed = true;
public String fc2livePassword = ""; public String fc2livePassword = "";
public String fc2liveUsername = ""; public String fc2liveUsername = "";
@ -79,7 +78,6 @@ public class Settings {
public String flirt4freeUsername; public String flirt4freeUsername;
public String fontFamily = "Sans-Serif"; public String fontFamily = "Sans-Serif";
public int fontSize = 14; public int fontSize = 14;
public boolean generatePlaylist = true;
public String hlsdlExecutable = "hlsdl"; public String hlsdlExecutable = "hlsdl";
public int httpPort = 8080; public int httpPort = 8080;
public int httpSecurePort = 8443; public int httpSecurePort = 8443;

93
common/src/main/java/ctbrec/recorder/AccuratePlaylistGenerator.java
View File

@ -1,93 +0,0 @@
package ctbrec.recorder;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FilenameFilter;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.io.Files;
import com.iheartradio.m3u8.Encoding;
import com.iheartradio.m3u8.Format;
import com.iheartradio.m3u8.ParseException;
import com.iheartradio.m3u8.ParsingMode;
import com.iheartradio.m3u8.PlaylistException;
import com.iheartradio.m3u8.PlaylistParser;
import com.iheartradio.m3u8.data.MediaPlaylist;
import com.iheartradio.m3u8.data.Playlist;
import com.iheartradio.m3u8.data.TrackData;
import com.iheartradio.m3u8.data.TrackInfo;
import ctbrec.MpegUtil;
public class AccuratePlaylistGenerator extends PlaylistGenerator {
private static final Logger LOG = LoggerFactory.getLogger(AccuratePlaylistGenerator.class);
AccuratePlaylistGenerator() {
}
@Override
public File generate(File directory, String fileSuffix) throws IOException, ParseException, PlaylistException {
LOG.info("Starting playlist generation for {}", directory);
File[] files = scanDirectoryForSegments(directory, fileSuffix);
// create a track containing all files
List<TrackData> track = new ArrayList<>();
int total = files.length;
int done = 0;
for (File file : files) {
try {
float duration = 0;
if (file.getName().toLowerCase().endsWith(".ts")) {
duration = (float) MpegUtil.getFileDurationInSecs(file);
if (duration <= 0) {
throw new InvalidTrackLengthException("Track has negative duration: " + file.getName());
}
}
track.add(new TrackData.Builder()
.withUri(file.getName())
.withTrackInfo(new TrackInfo(duration, file.getName()))
.build());
} catch (Exception | AssertionError e) {
LOG.warn("Couldn't determine duration for {}. Skipping this file.", file.getName());
File corruptedFile = new File(directory, file.getName() + ".corrupt");
Files.move(file, corruptedFile);
}
done++;
double percentage = (double) done / (double) total;
updateProgressListeners(percentage);
}
return writePlaylistFile(directory, track);
}
@Override
public void validate(File recDir) throws IOException, ParseException, PlaylistException {
File playlist = new File(recDir, "playlist.m3u8");
if (playlist.exists()) {
try (FileInputStream fin = new FileInputStream(playlist)) {
PlaylistParser playlistParser = new PlaylistParser(fin, Format.EXT_M3U, Encoding.UTF_8, ParsingMode.LENIENT);
Playlist m3u = playlistParser.parse();
MediaPlaylist mediaPlaylist = m3u.getMediaPlaylist();
int playlistSize = mediaPlaylist.getTracks().size();
File[] segments = recDir.listFiles((FilenameFilter) (dir, name) -> name.endsWith(".ts"));
if (segments.length == 0) {
throw new InvalidPlaylistException("No segments found. Playlist is empty");
} else if (segments.length != playlistSize) {
throw new InvalidPlaylistException("Playlist size and amount of segments differ (" + segments.length + " != " + playlistSize + ")");
} else {
LOG.debug("Generated playlist looks good");
}
}
} else {
throw new FileNotFoundException(playlist.getAbsolutePath() + " does not exist");
}
}
}

69
common/src/main/java/ctbrec/recorder/FastPlaylistGenerator.java
View File

@ -1,69 +0,0 @@
package ctbrec.recorder;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.iheartradio.m3u8.ParseException;
import com.iheartradio.m3u8.PlaylistException;
import com.iheartradio.m3u8.data.TrackData;
import com.iheartradio.m3u8.data.TrackInfo;
import ctbrec.MpegUtil;
public class FastPlaylistGenerator extends PlaylistGenerator {
static final Logger LOG = LoggerFactory.getLogger(FastPlaylistGenerator.class);
FastPlaylistGenerator() {
}
@Override
public File generate(File directory, String fileSuffix) throws IOException, ParseException, PlaylistException {
LOG.info("Starting playlist generation for {}", directory);
File[] files = scanDirectoryForSegments(directory, fileSuffix);
// create a track containing all files
List<TrackData> track = new ArrayList<>();
int total = files.length;
int done = 0;
float duration = getFileDuration(files);
for (File file : files) {
track.add(new TrackData.Builder()
.withUri(file.getName())
.withTrackInfo(new TrackInfo(duration, file.getName()))
.build());
done++;
double percentage = (double) done / (double) total;
updateProgressListeners(percentage);
}
return writePlaylistFile(directory, track);
}
private float getFileDuration(File[] files) {
for (File file : files) {
try {
float duration = 0;
if (file.getName().toLowerCase().endsWith(".ts")) {
duration = (float) MpegUtil.getFileDurationInSecs(file);
if (duration > 0) {
return duration;
}
}
} catch (Exception e) {
LOG.warn("Couldn't determine duration for {}. Skipping this file.", file.getName());
}
}
throw new RuntimeException("Couldn't determine playlist segment length");
}
@Override
public void validate(File recDir) throws IOException, ParseException, PlaylistException {
// don't validate anything
}
}

7
common/src/main/java/ctbrec/recorder/InvalidPlaylistException.java Normal file
View File

@ -0,0 +1,7 @@
package ctbrec.recorder;
public class InvalidPlaylistException extends RuntimeException {
public InvalidPlaylistException(String msg) {
super(msg);
}
}

7
common/src/main/java/ctbrec/recorder/InvalidTrackLengthException.java Normal file
View File

@ -0,0 +1,7 @@
package ctbrec.recorder;
public class InvalidTrackLengthException extends RuntimeException {
public InvalidTrackLengthException(String msg) {
super(msg);
}
}

129
common/src/main/java/ctbrec/recorder/PlaylistGenerator.java
View File

@ -1,129 +0,0 @@
package ctbrec.recorder;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.iheartradio.m3u8.Encoding;
import com.iheartradio.m3u8.Format;
import com.iheartradio.m3u8.ParseException;
import com.iheartradio.m3u8.PlaylistException;
import com.iheartradio.m3u8.PlaylistWriter;
import com.iheartradio.m3u8.data.MediaPlaylist;
import com.iheartradio.m3u8.data.Playlist;
import com.iheartradio.m3u8.data.PlaylistType;
import com.iheartradio.m3u8.data.TrackData;
public abstract class PlaylistGenerator {
private static final Logger LOG = LoggerFactory.getLogger(PlaylistGenerator.class);
private int lastPercentage;
private List<ProgressListener> listeners = new ArrayList<>();
public static PlaylistGenerator newInstance(boolean fast) {
if(fast) {
return new FastPlaylistGenerator();
} else {
return new AccuratePlaylistGenerator();
}
}
public File generate(File directory) throws IOException, ParseException, PlaylistException {
return generate(directory, "ts");
}
public abstract File generate(File directory, String fileSuffix) throws IOException, ParseException, PlaylistException;
public void addProgressListener(ProgressListener l) {
listeners.add(l);
}
public int getProgress() {
return lastPercentage;
}
protected void updateProgressListeners(double percentage) {
int p = (int) (percentage * 100);
if (p > lastPercentage) {
for (ProgressListener progressListener : listeners) {
progressListener.update(p);
}
lastPercentage = p;
}
}
public static class InvalidPlaylistException extends RuntimeException {
public InvalidPlaylistException(String msg) {
super(msg);
}
}
public static class InvalidTrackLengthException extends RuntimeException {
public InvalidTrackLengthException(String msg) {
super(msg);
}
}
public abstract void validate(File recDir) throws IOException, ParseException, PlaylistException;
protected File[] scanDirectoryForSegments(File directory, String fileSuffix) {
// get a list of all ts files and sort them by sequence
File[] files = directory.listFiles(f -> f.getName().endsWith('.' + fileSuffix));
if (files == null || files.length == 0) {
LOG.debug("{} is empty. Not going to generate a playlist", directory);
throw new InvalidPlaylistException("Directory is empty");
}
Arrays.sort(files, (f1, f2) -> {
String n1 = f1.getName();
String n2 = f2.getName();
return n1.compareTo(n2);
});
return files;
}
protected File writePlaylistFile(File directory, List<TrackData> track) throws IOException, ParseException, PlaylistException {
// create a media playlist
float targetDuration = getAvgDuration(track);
MediaPlaylist playlist = new MediaPlaylist.Builder()
.withPlaylistType(PlaylistType.VOD)
.withMediaSequenceNumber(0)
.withTargetDuration((int) targetDuration)
.withTracks(track).build();
// create a master playlist containing the media playlist
Playlist master = new Playlist.Builder()
.withCompatibilityVersion(4)
.withExtended(true)
.withMediaPlaylist(playlist)
.build();
// write the playlist to a file
File output = new File(directory, "playlist.m3u8");
try (FileOutputStream fos = new FileOutputStream(output)) {
PlaylistWriter writer = new PlaylistWriter.Builder()
.withFormat(Format.EXT_M3U)
.withEncoding(Encoding.UTF_8)
.withOutputStream(fos)
.build();
writer.write(master);
LOG.debug("Finished playlist generation for {}", directory);
}
return output;
}
private float getAvgDuration(List<TrackData> track) {
float targetDuration = 0;
for (TrackData trackData : track) {
targetDuration += trackData.getTrackInfo().duration;
}
targetDuration /= track.size();
return targetDuration;
}
}

6
common/src/main/java/ctbrec/recorder/download/AbstractDownload.java
View File

@ -19,9 +19,9 @@ public abstract class AbstractDownload implements Download {
protected Instant rescheduleTime = Instant.now(); protected Instant rescheduleTime = Instant.now();
protected Model model = new UnknownModel(); protected Model model = new UnknownModel();
protected transient Config config; protected Config config;
protected transient SplittingStrategy splittingStrategy; protected SplittingStrategy splittingStrategy;
protected transient ExecutorService downloadExecutor; protected ExecutorService downloadExecutor;
@Override @Override
public void init(Config config, Model model, Instant startTime, ExecutorService executorService) throws IOException { public void init(Config config, Model model, Instant startTime, ExecutorService executorService) throws IOException {

55
common/src/main/java/ctbrec/recorder/download/hls/AbstractHlsDownload.java
View File

@ -31,8 +31,10 @@ import java.util.Locale;
import java.util.Map; import java.util.Map;
import java.util.Map.Entry; import java.util.Map.Entry;
import java.util.Optional; import java.util.Optional;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorCompletionService;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import javax.xml.bind.JAXBException; import javax.xml.bind.JAXBException;
@ -59,10 +61,11 @@ import ctbrec.io.BandwidthMeter;
import ctbrec.io.HttpClient; import ctbrec.io.HttpClient;
import ctbrec.io.HttpConstants; import ctbrec.io.HttpConstants;
import ctbrec.io.HttpException; import ctbrec.io.HttpException;
import ctbrec.recorder.PlaylistGenerator.InvalidPlaylistException; import ctbrec.recorder.InvalidPlaylistException;
import ctbrec.recorder.download.AbstractDownload; import ctbrec.recorder.download.AbstractDownload;
import ctbrec.recorder.download.HttpHeaderFactory; import ctbrec.recorder.download.HttpHeaderFactory;
import ctbrec.recorder.download.StreamSource; import ctbrec.recorder.download.StreamSource;
import ctbrec.recorder.download.hls.SegmentPlaylist.Segment;
import ctbrec.sites.Site; import ctbrec.sites.Site;
import okhttp3.Request; import okhttp3.Request;
import okhttp3.Request.Builder; import okhttp3.Request.Builder;
@ -92,12 +95,19 @@ public abstract class AbstractHlsDownload extends AbstractDownload {
private int selectedResolution = UNKNOWN; private int selectedResolution = UNKNOWN;
private List<RecordingEvent> recordingEvents = new LinkedList<>(); private List<RecordingEvent> recordingEvents = new LinkedList<>();
protected ExecutorCompletionService<SegmentDownload> segmentDownloadService;
protected AbstractHlsDownload(HttpClient client) { protected AbstractHlsDownload(HttpClient client) {
this.client = client; this.client = client;
} }
protected abstract OutputStream getSegmentOutputStream(String prefix, String fileName) throws IOException; @Override
public void init(Config config, Model model, Instant startTime, ExecutorService executorService) throws IOException {
super.init(config, model, startTime, executorService);
segmentDownloadService = new ExecutorCompletionService<>(downloadExecutor);
}
protected abstract OutputStream getSegmentOutputStream(Segment segment) throws IOException;
protected void segmentDownloadFinished(SegmentDownload segmentDownload) { // NOSONAR protected void segmentDownloadFinished(SegmentDownload segmentDownload) { // NOSONAR
if (Duration.between(lastSegmentDownload, Instant.now()).getSeconds() > MAX_SECONDS_WITHOUT_TRANSFER) { if (Duration.between(lastSegmentDownload, Instant.now()).getSeconds() > MAX_SECONDS_WITHOUT_TRANSFER) {
@ -123,8 +133,9 @@ public abstract class AbstractHlsDownload extends AbstractDownload {
enqueueNewSegments(segmentPlaylist, nextSegmentNumber); enqueueNewSegments(segmentPlaylist, nextSegmentNumber);
splitRecordingIfNecessary(); splitRecordingIfNecessary();
calculateRescheduleTime(); calculateRescheduleTime();
processFinishedSegments();
// this if check makes sure, that we don't decrease nextSegment. for some reason // this if-check makes sure, that we don't decrease nextSegment. for some reason
// streamate playlists sometimes jump back. e.g. max sequence = 79 -> 80 -> 79 // streamate playlists sometimes jump back. e.g. max sequence = 79 -> 80 -> 79
lastSegmentNumber = segmentPlaylist.seq; lastSegmentNumber = segmentPlaylist.seq;
if (lastSegmentNumber + segmentPlaylist.segments.size() > nextSegmentNumber) { if (lastSegmentNumber + segmentPlaylist.segments.size() > nextSegmentNumber) {
@ -134,6 +145,7 @@ public abstract class AbstractHlsDownload extends AbstractDownload {
while (recordingEvents.size() > 30) { while (recordingEvents.size() > 30) {
recordingEvents.remove(0); recordingEvents.remove(0);
} }
} catch (ParseException e) { } catch (ParseException e) {
LOG.error("Couldn't parse HLS playlist for model {}\n{}", model, e.getInput(), e); LOG.error("Couldn't parse HLS playlist for model {}\n{}", model, e.getInput(), e);
stop(); stop();
@ -163,20 +175,24 @@ public abstract class AbstractHlsDownload extends AbstractDownload {
return this; return this;
} }
protected void execute(SegmentDownload segmentDownload) { protected void processFinishedSegments() {
CompletableFuture.supplyAsync(() -> downloadExecutor.submit(segmentDownload), downloadExecutor) downloadExecutor.submit((Runnable)() -> {
.whenComplete((result, executor) -> { Future<SegmentDownload> future;
try { while ((future = segmentDownloadService.poll()) != null) {
segmentDownloadFinished(result.get()); try {
} catch (InterruptedException e) { segmentDownloadFinished(future.get());
Thread.currentThread().interrupt(); } catch (InterruptedException e) {
LOG.error("Error in segmentDownloadFinished", e); Thread.currentThread().interrupt();
} catch (ExecutionException e) { LOG.error("Error in segmentDownloadFinished", e);
LOG.error("Error in segmentDownloadFinished", e); } catch (ExecutionException e) {
LOG.error("Error in segmentDownloadFinished", e);
}
} }
}); });
} }
protected abstract void execute(SegmentDownload segmentDownload);
protected void handleHttpException(HttpException e) throws IOException { protected void handleHttpException(HttpException e) throws IOException {
if (e.getResponseCode() == 404) { if (e.getResponseCode() == 404) {
checkIfModelIsStillOnline("Playlist not found (404). Model {} probably went offline. Model state: {}"); checkIfModelIsStillOnline("Playlist not found (404). Model {} probably went offline. Model state: {}");
@ -307,7 +323,7 @@ public abstract class AbstractHlsDownload extends AbstractDownload {
uri = new URL(context, uri).toExternalForm(); uri = new URL(context, uri).toExternalForm();
} }
lsp.totalDuration += trackData.getTrackInfo().duration; lsp.totalDuration += trackData.getTrackInfo().duration;
lsp.segments.add(uri); lsp.segments.add(new Segment(uri, trackData.getTrackInfo().duration));
if (trackData.hasEncryptionData()) { if (trackData.hasEncryptionData()) {
lsp.encrypted = true; lsp.encrypted = true;
EncryptionData data = trackData.getEncryptionData(); EncryptionData data = trackData.getEncryptionData();
@ -367,15 +383,14 @@ public abstract class AbstractHlsDownload extends AbstractDownload {
protected void enqueueNewSegments(SegmentPlaylist playlist, int nextSegmentNumber) throws IOException { protected void enqueueNewSegments(SegmentPlaylist playlist, int nextSegmentNumber) throws IOException {
int skip = nextSegmentNumber - playlist.seq; int skip = nextSegmentNumber - playlist.seq;
for (String segment : playlist.segments) { for (Segment segment : playlist.segments) {
if (skip > 0) { if (skip > 0) {
skip--; skip--;
} else { } else {
URL segmentUrl = new URL(segment);
String prefix = nf.format(segmentCounter++); String prefix = nf.format(segmentCounter++);
File tmp = new File(segmentUrl.getFile()); segment.prefix = prefix;
OutputStream targetStream = getSegmentOutputStream(prefix, tmp.getName()); OutputStream targetStream = getSegmentOutputStream(segment);
SegmentDownload segmentDownload = new SegmentDownload(model, playlist, segmentUrl, client, targetStream); SegmentDownload segmentDownload = new SegmentDownload(model, playlist, segment, client, targetStream);
execute(segmentDownload); execute(segmentDownload);
} }
} }

3
common/src/main/java/ctbrec/recorder/download/hls/FFmpegDownload.java
View File

@ -27,6 +27,7 @@ import ctbrec.Recording;
import ctbrec.io.HttpClient; import ctbrec.io.HttpClient;
import ctbrec.io.StreamRedirector; import ctbrec.io.StreamRedirector;
import ctbrec.recorder.download.ProcessExitedUncleanException; import ctbrec.recorder.download.ProcessExitedUncleanException;
import ctbrec.recorder.download.hls.SegmentPlaylist.Segment;
/** /**
* Does the whole HLS download with FFmpeg. Not used at the moment, because FFMpeg can't * Does the whole HLS download with FFmpeg. Not used at the moment, because FFMpeg can't
@ -162,7 +163,7 @@ public class FFmpegDownload extends AbstractHlsDownload {
} }
@Override @Override
protected OutputStream getSegmentOutputStream(String prefix, String fileName) throws IOException { protected OutputStream getSegmentOutputStream(Segment segment) throws IOException {
// TODO Auto-generated method stub // TODO Auto-generated method stub
return null; return null;
} }

142
common/src/main/java/ctbrec/recorder/download/hls/HlsDownload.java
View File

@ -5,6 +5,8 @@ import java.io.FileNotFoundException;
import java.io.FileOutputStream; import java.io.FileOutputStream;
import java.io.IOException; import java.io.IOException;
import java.io.OutputStream; import java.io.OutputStream;
import java.net.MalformedURLException;
import java.net.URL;
import java.nio.file.FileSystems; import java.nio.file.FileSystems;
import java.nio.file.Files; import java.nio.file.Files;
import java.nio.file.Path; import java.nio.file.Path;
@ -12,28 +14,45 @@ import java.time.Instant;
import java.time.ZoneId; import java.time.ZoneId;
import java.time.ZonedDateTime; import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter; import java.time.format.DateTimeFormatter;
import java.util.LinkedList;
import java.util.List;
import java.util.Queue;
import java.util.concurrent.ExecutorService; import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import com.iheartradio.m3u8.Encoding;
import com.iheartradio.m3u8.Format;
import com.iheartradio.m3u8.ParseException; import com.iheartradio.m3u8.ParseException;
import com.iheartradio.m3u8.PlaylistException; import com.iheartradio.m3u8.PlaylistException;
import com.iheartradio.m3u8.PlaylistWriter;
import com.iheartradio.m3u8.data.MediaPlaylist;
import com.iheartradio.m3u8.data.Playlist;
import com.iheartradio.m3u8.data.PlaylistType;
import com.iheartradio.m3u8.data.TrackData;
import com.iheartradio.m3u8.data.TrackInfo;
import ctbrec.Config; import ctbrec.Config;
import ctbrec.Model; import ctbrec.Model;
import ctbrec.Recording; import ctbrec.Recording;
import ctbrec.Recording.State;
import ctbrec.io.HttpClient; import ctbrec.io.HttpClient;
import ctbrec.io.IoUtils; import ctbrec.io.IoUtils;
import ctbrec.recorder.PlaylistGenerator; import ctbrec.recorder.download.hls.SegmentPlaylist.Segment;
public class HlsDownload extends AbstractHlsDownload { public class HlsDownload extends AbstractHlsDownload {
private static final Logger LOG = LoggerFactory.getLogger(HlsDownload.class); private static final Logger LOG = LoggerFactory.getLogger(HlsDownload.class);
protected transient Path downloadDir; protected Path downloadDir;
private Queue<Future<SegmentDownload>> segmentDownloads = new LinkedList<>();
private List<TrackData> segments = new LinkedList<>();
private float targetDuration;
public HlsDownload(HttpClient client) { public HlsDownload(HttpClient client) {
super(client); super(client);
@ -49,6 +68,83 @@ public class HlsDownload extends AbstractHlsDownload {
createTargetDirectory(); createTargetDirectory();
} }
@Override
public AbstractHlsDownload call() throws Exception {
super.call();
updatePlaylist();
return this;
}
@Override
protected SegmentPlaylist getNextSegments(String segmentPlaylistUrl) throws IOException, ParseException, PlaylistException {
SegmentPlaylist segmentPlaylist = super.getNextSegments(segmentPlaylistUrl);
targetDuration = segmentPlaylist.targetDuration;
return segmentPlaylist;
}
private void updatePlaylist() {
downloadExecutor.submit(() -> {
addNewSegmentsToPlaylist();
try {
MediaPlaylist playlist = new MediaPlaylist.Builder()
.withPlaylistType(PlaylistType.VOD)
.withMediaSequenceNumber(0)
.withTargetDuration(Math.round(targetDuration))
.withTracks(segments)
.build();
// create a master playlist containing the media playlist
Playlist master = new Playlist.Builder()
.withCompatibilityVersion(4)
.withExtended(true)
.withMediaPlaylist(playlist)
.build();
// write the playlist to a file
File output = new File(getTarget(), "playlist.m3u8");
try (FileOutputStream fos = new FileOutputStream(output)) {
PlaylistWriter writer = new PlaylistWriter.Builder()
.withFormat(Format.EXT_M3U)
.withEncoding(Encoding.UTF_8)
.withOutputStream(fos)
.build();
writer.write(master);
}
} catch (IOException | ParseException | PlaylistException e) {
LOG.error("Updating segment playlist failed", e);
}
LOG.trace("Segment queue size for {}: {}", model, segmentDownloads.size());
});
}
private void addNewSegmentsToPlaylist() {
Future<SegmentDownload> future;
while ((future = segmentDownloads.peek()) != null && !Thread.currentThread().isInterrupted()) {
try {
if (running && future.isDone()) {
segmentDownloads.poll(); // future is done remove from queue
SegmentDownload segmentDownload = future.get();
segments.add(toTrack(segmentDownload.getSegment()));
} else {
// first download in queue not finished, let's continue with other stuff
break;
}
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
} catch (Exception e) {
LOG.error("Segment download failed for model {}", model, e);
}
}
}
private TrackData toTrack(Segment segment) {
String filename = segment.targetFile.getName();
return new TrackData.Builder()
.withUri(filename)
.withTrackInfo(new TrackInfo(segment.duration, filename))
.build();
}
protected void createTargetDirectory() throws IOException { protected void createTargetDirectory() throws IOException {
if (!downloadDir.toFile().exists()) { if (!downloadDir.toFile().exists()) {
Files.createDirectories(downloadDir); Files.createDirectories(downloadDir);
@ -62,30 +158,7 @@ public class HlsDownload extends AbstractHlsDownload {
@Override @Override
public void postprocess(Recording recording) { public void postprocess(Recording recording) {
Thread.currentThread().setName("PP " + model.getName()); // nothing to do
recording.setStatusWithEvent(State.GENERATING_PLAYLIST);
try {
generatePlaylist(recording);
recording.setStatusWithEvent(State.POST_PROCESSING);
} catch (Exception e) {
throw new PostProcessingException(e);
}
}
protected File generatePlaylist(Recording recording) throws IOException, ParseException, PlaylistException {
File recDir = recording.getAbsoluteFile();
if (!config.getSettings().generatePlaylist) {
return null;
}
PlaylistGenerator playlistGenerator = PlaylistGenerator.newInstance(config.getSettings().fastPlaylistGenerator);
playlistGenerator.addProgressListener(recording::setProgress);
File playlist = playlistGenerator.generate(recDir);
if (playlist != null) {
playlistGenerator.validate(recDir);
}
recording.setProgress(-1);
return playlist;
} }
@Override @Override
@ -123,8 +196,10 @@ public class HlsDownload extends AbstractHlsDownload {
} }
@Override @Override
protected OutputStream getSegmentOutputStream(String prefix, String fileName) throws FileNotFoundException { protected OutputStream getSegmentOutputStream(Segment segment) throws FileNotFoundException, MalformedURLException {
String prefixedFileName = prefix + '_' + fileName; URL segmentUrl = new URL(segment.url);
File tmp = new File(segmentUrl.getFile());
String prefixedFileName = segment.prefix + '_' + tmp.getName();
int questionMarkPosition = prefixedFileName.indexOf('?'); int questionMarkPosition = prefixedFileName.indexOf('?');
if (questionMarkPosition > 0) { if (questionMarkPosition > 0) {
prefixedFileName = prefixedFileName.substring(0, questionMarkPosition); prefixedFileName = prefixedFileName.substring(0, questionMarkPosition);
@ -132,8 +207,8 @@ public class HlsDownload extends AbstractHlsDownload {
if (!prefixedFileName.endsWith(".ts")) { if (!prefixedFileName.endsWith(".ts")) {
prefixedFileName += ".ts"; prefixedFileName += ".ts";
} }
File file = FileSystems.getDefault().getPath(downloadDir.toAbsolutePath().toString(), prefixedFileName).toFile(); segment.targetFile = FileSystems.getDefault().getPath(downloadDir.toAbsolutePath().toString(), prefixedFileName).toFile();
return new FileOutputStream(file); return new FileOutputStream(segment.targetFile);
} }
@Override @Override
@ -141,4 +216,9 @@ public class HlsDownload extends AbstractHlsDownload {
super.segmentDownloadFinished(segmentDownload); super.segmentDownloadFinished(segmentDownload);
IoUtils.close(segmentDownload.getOutputStream(), "Couldn't close segment file"); IoUtils.close(segmentDownload.getOutputStream(), "Couldn't close segment file");
} }
@Override
protected void execute(SegmentDownload segmentDownload) {
segmentDownloads.add(segmentDownloadService.submit(segmentDownload));
}
} }

13
common/src/main/java/ctbrec/recorder/download/hls/MergedFfmpegHlsDownload.java
View File

@ -25,17 +25,18 @@ import ctbrec.Recording;
import ctbrec.io.HttpClient; import ctbrec.io.HttpClient;
import ctbrec.recorder.FFmpeg; import ctbrec.recorder.FFmpeg;
import ctbrec.recorder.download.ProcessExitedUncleanException; import ctbrec.recorder.download.ProcessExitedUncleanException;
import ctbrec.recorder.download.hls.SegmentPlaylist.Segment;
public class MergedFfmpegHlsDownload extends AbstractHlsDownload { public class MergedFfmpegHlsDownload extends AbstractHlsDownload {
private static final Logger LOG = LoggerFactory.getLogger(MergedFfmpegHlsDownload.class); private static final Logger LOG = LoggerFactory.getLogger(MergedFfmpegHlsDownload.class);
protected File targetFile; protected File targetFile;
protected transient FFmpeg ffmpeg; protected FFmpeg ffmpeg;
protected transient Process ffmpegProcess; protected Process ffmpegProcess;
protected transient OutputStream ffmpegStdIn; protected OutputStream ffmpegStdIn;
protected transient BlockingQueue<Future<SegmentDownload>> queue = new LinkedBlockingQueue<>(); protected BlockingQueue<Future<SegmentDownload>> queue = new LinkedBlockingQueue<>();
protected transient Lock ffmpegStreamLock = new ReentrantLock(); protected Lock ffmpegStreamLock = new ReentrantLock();
public MergedFfmpegHlsDownload(HttpClient client) { public MergedFfmpegHlsDownload(HttpClient client) {
super(client); super(client);
@ -212,7 +213,7 @@ public class MergedFfmpegHlsDownload extends AbstractHlsDownload {
} }
@Override @Override
protected OutputStream getSegmentOutputStream(String prefix, String fileName) throws IOException { protected OutputStream getSegmentOutputStream(Segment segment) throws IOException {
return new ByteArrayOutputStream(); return new ByteArrayOutputStream();
} }
} }

12
common/src/main/java/ctbrec/recorder/download/hls/SegmentDownload.java
View File

@ -7,6 +7,7 @@ import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.io.InterruptedIOException; import java.io.InterruptedIOException;
import java.io.OutputStream; import java.io.OutputStream;
import java.net.MalformedURLException;
import java.net.URL; import java.net.URL;
import java.security.InvalidAlgorithmParameterException; import java.security.InvalidAlgorithmParameterException;
import java.security.InvalidKeyException; import java.security.InvalidKeyException;
@ -25,6 +26,7 @@ import ctbrec.io.BandwidthMeter;
import ctbrec.io.HttpClient; import ctbrec.io.HttpClient;
import ctbrec.io.HttpException; import ctbrec.io.HttpException;
import ctbrec.recorder.download.HttpHeaderFactory; import ctbrec.recorder.download.HttpHeaderFactory;
import ctbrec.recorder.download.hls.SegmentPlaylist.Segment;
import okhttp3.Request; import okhttp3.Request;
import okhttp3.Request.Builder; import okhttp3.Request.Builder;
import okhttp3.Response; import okhttp3.Response;
@ -35,15 +37,17 @@ public class SegmentDownload implements Callable<SegmentDownload> {
private URL url; private URL url;
private HttpClient client; private HttpClient client;
private SegmentPlaylist playlist; private SegmentPlaylist playlist;
private Segment segment;
private Model model; private Model model;
private OutputStream out; private OutputStream out;
public SegmentDownload(Model model, SegmentPlaylist playlist, URL url, HttpClient client, OutputStream out) { public SegmentDownload(Model model, SegmentPlaylist playlist, Segment segment, HttpClient client, OutputStream out) throws MalformedURLException {
this.model = model; this.model = model;
this.playlist = playlist; this.playlist = playlist;
this.url = url; this.segment = segment;
this.client = client; this.client = client;
this.out = out; this.out = out;
this.url = new URL(segment.url);
} }
@Override @Override
@ -98,7 +102,7 @@ public class SegmentDownload implements Callable<SegmentDownload> {
return out; return out;
} }
public URL getUrl() { public Segment getSegment() {
return url; return segment;
} }
} }

15
common/src/main/java/ctbrec/recorder/download/hls/SegmentPlaylist.java
View File

@ -1,5 +1,6 @@
package ctbrec.recorder.download.hls; package ctbrec.recorder.download.hls;
import java.io.File;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
@ -9,7 +10,7 @@ public class SegmentPlaylist {
public float totalDuration = 0; public float totalDuration = 0;
public float avgSegDuration = 0; public float avgSegDuration = 0;
public float targetDuration = 0; public float targetDuration = 0;
public List<String> segments = new ArrayList<>(); public List<Segment> segments = new ArrayList<>();
public boolean encrypted = false; public boolean encrypted = false;
public String encryptionMethod = "AES-128"; public String encryptionMethod = "AES-128";
public String encryptionKeyUrl; public String encryptionKeyUrl;
@ -17,4 +18,16 @@ public class SegmentPlaylist {
public SegmentPlaylist(String url) { public SegmentPlaylist(String url) {
this.url = url; this.url = url;
} }
public static class Segment {
public String url;
public String prefix;
public File targetFile;
public float duration;
public Segment(String url, float duration) {
this.url = url;
this.duration = duration;
}
}
} }

2
server/src/main/java/ctbrec/recorder/server/ConfigServlet.java
View File

@ -57,8 +57,6 @@ public class ConfigServlet extends AbstractCtbrecServlet {
addParameter("httpSecurePort", "HTTPS port", DataType.INTEGER, settings.httpSecurePort, json); addParameter("httpSecurePort", "HTTPS port", DataType.INTEGER, settings.httpSecurePort, json);
addParameter("httpUserAgent", "User-Agent", DataType.STRING, settings.httpUserAgent, json); addParameter("httpUserAgent", "User-Agent", DataType.STRING, settings.httpUserAgent, json);
addParameter("httpUserAgentMobile", "Mobile User-Agent", DataType.STRING, settings.httpUserAgentMobile, json); addParameter("httpUserAgentMobile", "Mobile User-Agent", DataType.STRING, settings.httpUserAgentMobile, json);
addParameter("generatePlaylist", "Generate Playlist", DataType.BOOLEAN, settings.generatePlaylist, json);
addParameter("fastPlaylistGenerator", "Use Fast Playlist Generator", DataType.BOOLEAN, settings.fastPlaylistGenerator, json);
addParameter("minimumResolution", "Minimum Resolution", DataType.INTEGER, settings.minimumResolution, json); addParameter("minimumResolution", "Minimum Resolution", DataType.INTEGER, settings.minimumResolution, json);
addParameter("maximumResolution", "Maximum Resolution", DataType.INTEGER, settings.maximumResolution, json); addParameter("maximumResolution", "Maximum Resolution", DataType.INTEGER, settings.maximumResolution, json);
addParameter("minimumSpaceLeftInBytes", "Leave Space On Device (bytes)", DataType.LONG, settings.minimumSpaceLeftInBytes, json); addParameter("minimumSpaceLeftInBytes", "Leave Space On Device (bytes)", DataType.LONG, settings.minimumSpaceLeftInBytes, json);