jafea7
/
ctbrec-5.3.2-experimental
mirror of https://github.com/Jafea7/ctbrec-5.3.2-original.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Run post-processing steps in runnable in a thread pool 

Server and client now create a runnable for post-processing steps,
which run in a thread pool. This ensures, that the steps run linearly so
that RecordingStateChange events make sense, too.
This commit is contained in:
0xboobface 2018-12-10 15:27:56 +01:00
parent 5b15b77014
commit 1d409fa1d4
1 changed files with 77 additions and 113 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

190
common/src/main/java/ctbrec/recorder/LocalRecorder.java
View File

@ -24,6 +24,8 @@ import java.util.Map;
import java.util.Map.Entry; import java.util.Map.Entry;
import java.util.NoSuchElementException; import java.util.NoSuchElementException;
import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.ReentrantLock; import java.util.concurrent.locks.ReentrantLock;
import java.util.stream.Collectors; import java.util.stream.Collectors;
@ -62,13 +64,14 @@ public class LocalRecorder implements Recorder {
private Map<File, PlaylistGenerator> playlistGenerators = new HashMap<>(); private Map<File, PlaylistGenerator> playlistGenerators = new HashMap<>();
private Config config; private Config config;
private ProcessMonitor processMonitor; private ProcessMonitor processMonitor;
private PostProcessingTrigger postProcessingTrigger;
private volatile boolean recording = true; private volatile boolean recording = true;
private List<File> deleteInProgress = Collections.synchronizedList(new ArrayList<>()); private List<File> deleteInProgress = Collections.synchronizedList(new ArrayList<>());
private RecorderHttpClient client = new RecorderHttpClient(); private RecorderHttpClient client = new RecorderHttpClient();
private ReentrantLock lock = new ReentrantLock(); private ReentrantLock lock = new ReentrantLock();
private long lastSpaceMessage = 0; private long lastSpaceMessage = 0;
private ExecutorService ppThreadPool = Executors.newFixedThreadPool(2);
public LocalRecorder(Config config) { public LocalRecorder(Config config) {
this.config = config; this.config = config;
config.getSettings().models.stream().forEach((m) -> { config.getSettings().models.stream().forEach((m) -> {
@ -83,12 +86,10 @@ public class LocalRecorder implements Recorder {
processMonitor = new ProcessMonitor(); processMonitor = new ProcessMonitor();
processMonitor.start(); processMonitor.start();
postProcessingTrigger = new PostProcessingTrigger();
if(Config.isServerMode()) {
postProcessingTrigger.start();
}
registerEventBusListener(); registerEventBusListener();
if(Config.isServerMode()) {
processUnfinishedRecordings();
}
LOG.debug("Recorder initialized"); LOG.debug("Recorder initialized");
LOG.info("Models to record: {}", models); LOG.info("Models to record: {}", models);
@ -206,47 +207,40 @@ public class LocalRecorder implements Recorder {
Download download = recordingProcesses.get(model); Download download = recordingProcesses.get(model);
download.stop(); download.stop();
recordingProcesses.remove(model); recordingProcesses.remove(model);
if(!Config.isServerMode()) { fireRecordingStateChanged(download.getTarget(), STOPPED, model, download.getStartTime());
postprocess(download); ppThreadPool.submit(createPostProcessor(download));
}
fireRecordingStateChanged(download.getTarget(), FINISHED, model, download.getStartTime());
} }
private void postprocess(Download download) { private void postprocess(Download download) {
if(!(download instanceof MergedHlsDownload)) {
throw new IllegalArgumentException("Download should be of type MergedHlsDownload");
}
String postProcessing = Config.getInstance().getSettings().postProcessing; String postProcessing = Config.getInstance().getSettings().postProcessing;
if (postProcessing != null && !postProcessing.isEmpty()) { if (postProcessing != null && !postProcessing.isEmpty()) {
new Thread(() -> { Runtime rt = Runtime.getRuntime();
Runtime rt = Runtime.getRuntime(); try {
try { String[] args = new String[] {
MergedHlsDownload d = (MergedHlsDownload) download; postProcessing,
String[] args = new String[] { download.getTarget().getParentFile().getAbsolutePath(),
postProcessing, download.getTarget().getAbsolutePath(),
d.getTarget().getParentFile().getAbsolutePath(), download.getModel().getName(),
d.getTarget().getAbsolutePath(), download.getModel().getSite().getName(),
d.getModel().getName(), Long.toString(download.getStartTime().getEpochSecond())
d.getModel().getSite().getName(), };
Long.toString(download.getStartTime().getEpochSecond()) LOG.debug("Running {}", Arrays.toString(args));
}; Process process = rt.exec(args, OS.getEnvironment());
LOG.debug("Running {}", Arrays.toString(args)); // TODO maybe write these to a separate log file, e.g. recname.ts.pp.log
Process process = rt.exec(args, OS.getEnvironment()); Thread std = new Thread(new StreamRedirectThread(process.getInputStream(), System.out));
Thread std = new Thread(new StreamRedirectThread(process.getInputStream(), System.out)); std.setName("Process stdout pipe");
std.setName("Process stdout pipe"); std.setDaemon(true);
std.setDaemon(true); std.start();
std.start(); Thread err = new Thread(new StreamRedirectThread(process.getErrorStream(), System.err));
Thread err = new Thread(new StreamRedirectThread(process.getErrorStream(), System.err)); err.setName("Process stderr pipe");
err.setName("Process stderr pipe"); err.setDaemon(true);
err.setDaemon(true); err.start();
err.start();
process.waitFor(); process.waitFor();
LOG.debug("Process finished."); LOG.debug("Process finished.");
} catch (Exception e) { } catch (Exception e) {
LOG.error("Error in process thread", e); LOG.error("Error in process thread", e);
} }
}).start();
} }
} }
@ -306,9 +300,9 @@ public class LocalRecorder implements Recorder {
recording = false; recording = false;
LOG.debug("Stopping monitor threads"); LOG.debug("Stopping monitor threads");
processMonitor.running = false; processMonitor.running = false;
postProcessingTrigger.running = false;
LOG.debug("Stopping all recording processes"); LOG.debug("Stopping all recording processes");
stopRecordingProcesses(); stopRecordingProcesses();
ppThreadPool.shutdown();
client.shutdown(); client.shutdown();
} }
@ -318,12 +312,7 @@ public class LocalRecorder implements Recorder {
for (Model model : models) { for (Model model : models) {
Download recordingProcess = recordingProcesses.get(model); Download recordingProcess = recordingProcesses.get(model);
if (recordingProcess != null) { if (recordingProcess != null) {
try { stopRecordingProcess(model);
recordingProcess.stop();
LOG.debug("Stopped recording for {}", model);
} catch (Exception e) {
LOG.error("Couldn't stop recording for model {}", model, e);
}
} }
} }
} finally { } finally {
@ -375,21 +364,14 @@ public class LocalRecorder implements Recorder {
for (Iterator<Entry<Model, Download>> iterator = recordingProcesses.entrySet().iterator(); iterator.hasNext();) { for (Iterator<Entry<Model, Download>> iterator = recordingProcesses.entrySet().iterator(); iterator.hasNext();) {
Entry<Model, Download> entry = iterator.next(); Entry<Model, Download> entry = iterator.next();
Model m = entry.getKey(); Model m = entry.getKey();
Download d = entry.getValue(); Download download = entry.getValue();
if (!d.isAlive()) { if (!download.isAlive()) {
LOG.debug("Recording terminated for model {}", m.getName()); LOG.debug("Recording terminated for model {}", m.getName());
iterator.remove(); iterator.remove();
restart.add(m); restart.add(m);
if(Config.isServerMode()) { fireRecordingStateChanged(download.getTarget(), STOPPED, m, download.getStartTime());
try { Runnable pp = createPostProcessor(download);
finishRecording(d.getTarget()); ppThreadPool.submit(pp);
} catch(Exception e) {
LOG.error("Error while finishing recording for model {}", m.getName(), e);
}
} else {
postprocess(d);
}
fireRecordingStateChanged(d.getTarget(), FINISHED, m, d.getStartTime()); // TODO fire all the events
} }
} }
for (Model m : restart) { for (Model m : restart) {
@ -407,20 +389,6 @@ public class LocalRecorder implements Recorder {
} }
} }
private void finishRecording(File directory) {
if(Config.isServerMode()) {
Thread t = new Thread() {
@Override
public void run() {
generatePlaylist(directory);
}
};
t.setDaemon(true);
t.setName("Post-Processing " + directory.toString());
t.start();
}
}
private void generatePlaylist(File recDir) { private void generatePlaylist(File recDir) {
PlaylistGenerator playlistGenerator = new PlaylistGenerator(); PlaylistGenerator playlistGenerator = new PlaylistGenerator();
playlistGenerators.put(recDir, playlistGenerator); playlistGenerators.put(recDir, playlistGenerator);
@ -445,49 +413,32 @@ public class LocalRecorder implements Recorder {
EventBusHolder.BUS.post(evt); EventBusHolder.BUS.post(evt);
} }
private class PostProcessingTrigger extends Thread { /**
private volatile boolean running = false; * This is called once at start for server mode. When the server is killed, recordings are
* left without playlist. This method creates playlists for them.
public PostProcessingTrigger() { */
setName("PostProcessingTrigger"); private void processUnfinishedRecordings() {
setDaemon(true); try {
} List<Recording> recs = getRecordings();
for (Recording rec : recs) {
@Override if (rec.getStatus() == RECORDING) {
public void run() { boolean recordingProcessFound = false;
running = true; File recordingsDir = new File(config.getSettings().recordingsDir);
while (running) { File recDir = new File(recordingsDir, rec.getPath());
try { for (Entry<Model, Download> download : recordingProcesses.entrySet()) {
List<Recording> recs = getRecordings(); if (download.getValue().getTarget().equals(recDir)) {
for (Recording rec : recs) { recordingProcessFound = true;
if (rec.getStatus() == RECORDING) {
boolean recordingProcessFound = false;
File recordingsDir = new File(config.getSettings().recordingsDir);
File recDir = new File(recordingsDir, rec.getPath());
for (Entry<Model, Download> download : recordingProcesses.entrySet()) {
if (download.getValue().getTarget().equals(recDir)) {
recordingProcessFound = true;
}
}
if (!recordingProcessFound) {
if (deleteInProgress.contains(recDir)) {
LOG.debug("{} is being deleted. Not going to start post-processing", recDir);
} else {
finishRecording(recDir);
}
}
} }
} }
if (!recordingProcessFound) {
if (running) ppThreadPool.submit(() -> {
Thread.sleep(10000); generatePlaylist(recDir);
} catch (InterruptedException e) { });
LOG.error("Couldn't sleep", e); }
} catch (Exception e) {
LOG.error("Unexpected error in playlist trigger thread", e);
} }
} }
LOG.debug(getName() + " terminated"); } catch (Exception e) {
LOG.error("Unexpected error in playlist trigger", e);
} }
} }
@ -781,4 +732,17 @@ public class LocalRecorder implements Recorder {
return getFreeSpaceBytes() > minimum; return getFreeSpaceBytes() > minimum;
} }
} }
private Runnable createPostProcessor(Download download) {
return () -> {
LOG.debug("Starting post-processing for {}", download.getTarget());
if(Config.isServerMode()) {
fireRecordingStateChanged(download.getTarget(), GENERATING_PLAYLIST, download.getModel(), download.getStartTime());
generatePlaylist(download.getTarget());
}
fireRecordingStateChanged(download.getTarget(), POST_PROCESSING, download.getModel(), download.getStartTime());
postprocess(download);
fireRecordingStateChanged(download.getTarget(), FINISHED, download.getModel(), download.getStartTime());
};
}
} }