davebluke
/
ctbrec
forked from j62/ctbrec
1
0
Fork 0
Code Pull Requests Activity

Add switch to choose between fast and accurate playlist generation

This commit is contained in:
0xb00bface 2020-12-20 20:13:34 +01:00
parent a5834e3533
commit 3b8022df87
8 changed files with 215 additions and 95 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGELOG.md
View File

@ -3,6 +3,7 @@
* Fixed MVLive recordings once again
* Fix: "Check URLs" button stays inactive after the first run
* Fix: recordings for some Cam4 models still didn't start
* Added server setting to choose between fast and accurate playlist generation
* Some smaller tweaks here and there
3.10.9

2
client/src/main/resources/html/docs/ConfigurationFile.md
View File

@ -24,6 +24,8 @@ until a recording is finished. 0 means unlimited.
- **determineResolution** (app only) - [`true`,`false`] Display the stream resolution on the thumbnails.
- **fastPlaylistGenerator** (server only) - Use a fast playlist generator, which is not as accurate. This might lead to inaccurate skipping and timelines in media players. Useful for weak devices.
- **generatePlaylist** (server only) - [`true`,`false`] Generate a playlist once a recording terminates.
- **httpPort** - [1 - 65536] The TCP port, the server listens on. In the server config, this will tell the server, which port to use. In the application this will set

1
common/src/main/java/ctbrec/Settings.java
View File

@ -59,6 +59,7 @@ public class Settings {
public List<String> disabledSites = new ArrayList<>();
public String downloadFilename = "${modelSanitizedName}-${localDateTime}";
public List<EventHandlerConfiguration> eventHandlers = new ArrayList<>();
public boolean fastPlaylistGenerator = false;
public String fc2livePassword = "";
public String fc2liveUsername = "";
public String ffmpegMergedDownloadArgs = "-c:v copy -c:a copy -movflags faststart -y -f mpegts";

93
common/src/main/java/ctbrec/recorder/AccuratePlaylistGenerator.java Normal file
View File

@ -0,0 +1,93 @@
package ctbrec.recorder;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FilenameFilter;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.io.Files;
import com.iheartradio.m3u8.Encoding;
import com.iheartradio.m3u8.Format;
import com.iheartradio.m3u8.ParseException;
import com.iheartradio.m3u8.ParsingMode;
import com.iheartradio.m3u8.PlaylistException;
import com.iheartradio.m3u8.PlaylistParser;
import com.iheartradio.m3u8.data.MediaPlaylist;
import com.iheartradio.m3u8.data.Playlist;
import com.iheartradio.m3u8.data.TrackData;
import com.iheartradio.m3u8.data.TrackInfo;
import ctbrec.MpegUtil;
public class AccuratePlaylistGenerator extends PlaylistGenerator {
private static final Logger LOG = LoggerFactory.getLogger(AccuratePlaylistGenerator.class);
AccuratePlaylistGenerator() {
}
@Override
public File generate(File directory, String fileSuffix) throws IOException, ParseException, PlaylistException {
LOG.info("Starting playlist generation for {}", directory);
File[] files = scanDirectoryForSegments(directory, fileSuffix);
// create a track containing all files
List<TrackData> track = new ArrayList<>();
int total = files.length;
int done = 0;
for (File file : files) {
try {
float duration = 0;
if (file.getName().toLowerCase().endsWith(".ts")) {
duration = (float) MpegUtil.getFileDurationInSecs(file);
if (duration <= 0) {
throw new InvalidTrackLengthException("Track has negative duration: " + file.getName());
}
}
track.add(new TrackData.Builder()
.withUri(file.getName())
.withTrackInfo(new TrackInfo(duration, file.getName()))
.build());
} catch (Exception e) {
LOG.warn("Couldn't determine duration for {}. Skipping this file.", file.getName());
File corruptedFile = new File(directory, file.getName() + ".corrupt");
Files.move(file, corruptedFile);
}
done++;
double percentage = (double) done / (double) total;
updateProgressListeners(percentage);
}
return writePlaylistFile(directory, track);
}
@Override
public void validate(File recDir) throws IOException, ParseException, PlaylistException {
File playlist = new File(recDir, "playlist.m3u8");
if (playlist.exists()) {
try (FileInputStream fin = new FileInputStream(playlist)) {
PlaylistParser playlistParser = new PlaylistParser(fin, Format.EXT_M3U, Encoding.UTF_8, ParsingMode.LENIENT);
Playlist m3u = playlistParser.parse();
MediaPlaylist mediaPlaylist = m3u.getMediaPlaylist();
int playlistSize = mediaPlaylist.getTracks().size();
File[] segments = recDir.listFiles((FilenameFilter) (dir, name) -> name.endsWith(".ts"));
if (segments.length == 0) {
throw new InvalidPlaylistException("No segments found. Playlist is empty");
} else if (segments.length != playlistSize) {
throw new InvalidPlaylistException("Playlist size and amount of segments differ (" + segments.length + " != " + playlistSize + ")");
} else {
LOG.debug("Generated playlist looks good");
}
}
} else {
throw new FileNotFoundException(playlist.getAbsolutePath() + " does not exist");
}
}
}

69
common/src/main/java/ctbrec/recorder/FastPlaylistGenerator.java Normal file
View File

@ -0,0 +1,69 @@
package ctbrec.recorder;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.iheartradio.m3u8.ParseException;
import com.iheartradio.m3u8.PlaylistException;
import com.iheartradio.m3u8.data.TrackData;
import com.iheartradio.m3u8.data.TrackInfo;
import ctbrec.MpegUtil;
public class FastPlaylistGenerator extends PlaylistGenerator {
static final Logger LOG = LoggerFactory.getLogger(FastPlaylistGenerator.class);
FastPlaylistGenerator() {
}
@Override
public File generate(File directory, String fileSuffix) throws IOException, ParseException, PlaylistException {
LOG.info("Starting playlist generation for {}", directory);
File[] files = scanDirectoryForSegments(directory, fileSuffix);
// create a track containing all files
List<TrackData> track = new ArrayList<>();
int total = files.length;
int done = 0;
float duration = getFileDuration(files);
for (File file : files) {
track.add(new TrackData.Builder()
.withUri(file.getName())
.withTrackInfo(new TrackInfo(duration, file.getName()))
.build());
done++;
double percentage = (double) done / (double) total;
updateProgressListeners(percentage);
}
return writePlaylistFile(directory, track);
}
private float getFileDuration(File[] files) {
for (File file : files) {
try {
float duration = 0;
if (file.getName().toLowerCase().endsWith(".ts")) {
duration = (float) MpegUtil.getFileDurationInSecs(file);
if (duration > 0) {
return duration;
}
}
} catch (Exception e) {
LOG.warn("Couldn't determine duration for {}. Skipping this file.", file.getName());
}
}
throw new RuntimeException("Couldn't determine playlist segment length");
}
@Override
public void validate(File recDir) throws IOException, ParseException, PlaylistException {
// don't validate anything
}
}

140
common/src/main/java/ctbrec/recorder/PlaylistGenerator.java
View File

@ -1,10 +1,7 @@
package ctbrec.recorder;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FilenameFilter;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
@ -13,35 +10,69 @@ import java.util.List;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.io.Files;
import com.iheartradio.m3u8.Encoding;
import com.iheartradio.m3u8.Format;
import com.iheartradio.m3u8.ParseException;
import com.iheartradio.m3u8.ParsingMode;
import com.iheartradio.m3u8.PlaylistException;
import com.iheartradio.m3u8.PlaylistParser;
import com.iheartradio.m3u8.PlaylistWriter;
import com.iheartradio.m3u8.data.MediaPlaylist;
import com.iheartradio.m3u8.data.Playlist;
import com.iheartradio.m3u8.data.PlaylistType;
import com.iheartradio.m3u8.data.TrackData;
import com.iheartradio.m3u8.data.TrackInfo;
import ctbrec.MpegUtil;
public abstract class PlaylistGenerator {
public class PlaylistGenerator {
private static final Logger LOG = LoggerFactory.getLogger(PlaylistGenerator.class);
private int lastPercentage;
private List<ProgressListener> listeners = new ArrayList<>();
public static PlaylistGenerator newInstance(boolean fast) {
if(fast) {
return new FastPlaylistGenerator();
} else {
return new AccuratePlaylistGenerator();
}
}
public File generate(File directory) throws IOException, ParseException, PlaylistException {
return generate(directory, "ts");
}
public File generate(File directory, String fileSuffix) throws IOException, ParseException, PlaylistException {
LOG.info("Starting playlist generation for {}", directory);
public abstract File generate(File directory, String fileSuffix) throws IOException, ParseException, PlaylistException;
public void addProgressListener(ProgressListener l) {
listeners.add(l);
}
public int getProgress() {
return lastPercentage;
}
protected void updateProgressListeners(double percentage) {
int p = (int) (percentage * 100);
if (p > lastPercentage) {
for (ProgressListener progressListener : listeners) {
progressListener.update(p);
}
lastPercentage = p;
}
}
public static class InvalidPlaylistException extends RuntimeException {
public InvalidPlaylistException(String msg) {
super(msg);
}
}
public static class InvalidTrackLengthException extends RuntimeException {
public InvalidTrackLengthException(String msg) {
super(msg);
}
}
public abstract void validate(File recDir) throws IOException, ParseException, PlaylistException;
protected File[] scanDirectoryForSegments(File directory, String fileSuffix) {
// get a list of all ts files and sort them by sequence
File[] files = directory.listFiles(f -> f.getName().endsWith('.' + fileSuffix));
if (files == null || files.length == 0) {
@ -54,35 +85,10 @@ public class PlaylistGenerator {
String n2 = f2.getName();
return n1.compareTo(n2);
});
return files;
}
// create a track containing all files
List<TrackData> track = new ArrayList<>();
int total = files.length;
int done = 0;
for (File file : files) {
try {
float duration = 0;
if (file.getName().toLowerCase().endsWith(".ts")) {
duration = (float) MpegUtil.getFileDurationInSecs(file);
if (duration <= 0) {
throw new InvalidTrackLengthException("Track has negative duration: " + file.getName());
}
}
track.add(new TrackData.Builder()
.withUri(file.getName())
.withTrackInfo(new TrackInfo(duration, file.getName()))
.build());
} catch (Exception e) {
LOG.warn("Couldn't determine duration for {}. Skipping this file.", file.getName());
File corruptedFile = new File(directory, file.getName() + ".corrupt");
Files.move(file, corruptedFile);
}
done++;
double percentage = (double) done / (double) total;
updateProgressListeners(percentage);
}
protected File writePlaylistFile(File directory, List<TrackData> track) throws IOException, ParseException, PlaylistException {
// create a media playlist
float targetDuration = getAvgDuration(track);
MediaPlaylist playlist = new MediaPlaylist.Builder()
@ -112,16 +118,6 @@ public class PlaylistGenerator {
return output;
}
private void updateProgressListeners(double percentage) {
int p = (int) (percentage * 100);
if (p > lastPercentage) {
for (ProgressListener progressListener : listeners) {
progressListener.update(p);
}
lastPercentage = p;
}
}
private float getAvgDuration(List<TrackData> track) {
float targetDuration = 0;
for (TrackData trackData : track) {
@ -130,46 +126,4 @@ public class PlaylistGenerator {
targetDuration /= track.size();
return targetDuration;
}
public void addProgressListener(ProgressListener l) {
listeners.add(l);
}
public int getProgress() {
return lastPercentage;
}
public void validate(File recDir) throws IOException, ParseException, PlaylistException {
File playlist = new File(recDir, "playlist.m3u8");
if (playlist.exists()) {
try (FileInputStream fin = new FileInputStream(playlist)) {
PlaylistParser playlistParser = new PlaylistParser(fin, Format.EXT_M3U, Encoding.UTF_8, ParsingMode.LENIENT);
Playlist m3u = playlistParser.parse();
MediaPlaylist mediaPlaylist = m3u.getMediaPlaylist();
int playlistSize = mediaPlaylist.getTracks().size();
File[] segments = recDir.listFiles((FilenameFilter) (dir, name) -> name.endsWith(".ts"));
if (segments.length == 0) {
throw new InvalidPlaylistException("No segments found. Playlist is empty");
} else if (segments.length != playlistSize) {
throw new InvalidPlaylistException("Playlist size and amount of segments differ (" + segments.length + " != " + playlistSize + ")");
} else {
LOG.debug("Generated playlist looks good");
}
}
} else {
throw new FileNotFoundException(playlist.getAbsolutePath() + " does not exist");
}
}
public static class InvalidPlaylistException extends RuntimeException {
public InvalidPlaylistException(String msg) {
super(msg);
}
}
public static class InvalidTrackLengthException extends RuntimeException {
public InvalidTrackLengthException(String msg) {
super(msg);
}
}
}

2
common/src/main/java/ctbrec/recorder/download/hls/HlsDownload.java
View File

@ -238,7 +238,7 @@ public class HlsDownload extends AbstractHlsDownload {
if (!config.getSettings().generatePlaylist) {
return null;
}
PlaylistGenerator playlistGenerator = new PlaylistGenerator();
PlaylistGenerator playlistGenerator = PlaylistGenerator.newInstance(config.getSettings().fastPlaylistGenerator);
playlistGenerator.addProgressListener(recording::setProgress);
File playlist = playlistGenerator.generate(recDir);
if (playlist != null) {

2
common/src/main/java/ctbrec/sites/showup/ShowupDownload.java
View File

@ -23,7 +23,7 @@ public class ShowupDownload extends HlsDownload {
if (!config.getSettings().generatePlaylist) {
return null;
}
PlaylistGenerator playlistGenerator = new PlaylistGenerator();
PlaylistGenerator playlistGenerator = PlaylistGenerator.newInstance(config.getSettings().fastPlaylistGenerator);
playlistGenerator.addProgressListener(recording::setProgress);
File playlist = playlistGenerator.generate(recDir, "mp4");
recording.setProgress(-1);