jafea7
/
ctbrec-5.3.2-experimental
mirror of https://github.com/Jafea7/ctbrec-5.3.2-original.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Run post-processing steps in runnable in a thread pool 

Server and client now create a runnable for post-processing steps,
which run in a thread pool. This ensures, that the steps run linearly so
that RecordingStateChange events make sense, too.
This commit is contained in:
0xboobface 2018-12-10 15:27:56 +01:00
parent 5b15b77014
commit 1d409fa1d4
1 changed files with 77 additions and 113 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

122
common/src/main/java/ctbrec/recorder/LocalRecorder.java
View File

@ -24,6 +24,8 @@ import java.util.Map;
import java.util.Map.Entry; import java.util.Map.Entry;
import java.util.NoSuchElementException; import java.util.NoSuchElementException;
import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.ReentrantLock; import java.util.concurrent.locks.ReentrantLock;
import java.util.stream.Collectors; import java.util.stream.Collectors;
@ -62,13 +64,14 @@ public class LocalRecorder implements Recorder {
private Map<File, PlaylistGenerator> playlistGenerators = new HashMap<>(); private Map<File, PlaylistGenerator> playlistGenerators = new HashMap<>();
private Config config; private Config config;
private ProcessMonitor processMonitor; private ProcessMonitor processMonitor;
private PostProcessingTrigger postProcessingTrigger;
private volatile boolean recording = true; private volatile boolean recording = true;
private List<File> deleteInProgress = Collections.synchronizedList(new ArrayList<>()); private List<File> deleteInProgress = Collections.synchronizedList(new ArrayList<>());
private RecorderHttpClient client = new RecorderHttpClient(); private RecorderHttpClient client = new RecorderHttpClient();
private ReentrantLock lock = new ReentrantLock(); private ReentrantLock lock = new ReentrantLock();
private long lastSpaceMessage = 0; private long lastSpaceMessage = 0;
private ExecutorService ppThreadPool = Executors.newFixedThreadPool(2);
public LocalRecorder(Config config) { public LocalRecorder(Config config) {
this.config = config; this.config = config;
config.getSettings().models.stream().forEach((m) -> { config.getSettings().models.stream().forEach((m) -> {
@ -83,12 +86,10 @@ public class LocalRecorder implements Recorder {
processMonitor = new ProcessMonitor(); processMonitor = new ProcessMonitor();
processMonitor.start(); processMonitor.start();
postProcessingTrigger = new PostProcessingTrigger();
if(Config.isServerMode()) {
postProcessingTrigger.start();
}
registerEventBusListener(); registerEventBusListener();
if(Config.isServerMode()) {
processUnfinishedRecordings();
}
LOG.debug("Recorder initialized"); LOG.debug("Recorder initialized");
LOG.info("Models to record: {}", models); LOG.info("Models to record: {}", models);
@ -206,32 +207,26 @@ public class LocalRecorder implements Recorder {
Download download = recordingProcesses.get(model); Download download = recordingProcesses.get(model);
download.stop(); download.stop();
recordingProcesses.remove(model); recordingProcesses.remove(model);
if(!Config.isServerMode()) { fireRecordingStateChanged(download.getTarget(), STOPPED, model, download.getStartTime());
postprocess(download); ppThreadPool.submit(createPostProcessor(download));
}
fireRecordingStateChanged(download.getTarget(), FINISHED, model, download.getStartTime());
} }
private void postprocess(Download download) { private void postprocess(Download download) {
if(!(download instanceof MergedHlsDownload)) {
throw new IllegalArgumentException("Download should be of type MergedHlsDownload");
}
String postProcessing = Config.getInstance().getSettings().postProcessing; String postProcessing = Config.getInstance().getSettings().postProcessing;
if (postProcessing != null && !postProcessing.isEmpty()) { if (postProcessing != null && !postProcessing.isEmpty()) {
new Thread(() -> {
Runtime rt = Runtime.getRuntime(); Runtime rt = Runtime.getRuntime();
try { try {
MergedHlsDownload d = (MergedHlsDownload) download;
String[] args = new String[] { String[] args = new String[] {
postProcessing, postProcessing,
d.getTarget().getParentFile().getAbsolutePath(), download.getTarget().getParentFile().getAbsolutePath(),
d.getTarget().getAbsolutePath(), download.getTarget().getAbsolutePath(),
d.getModel().getName(), download.getModel().getName(),
d.getModel().getSite().getName(), download.getModel().getSite().getName(),
Long.toString(download.getStartTime().getEpochSecond()) Long.toString(download.getStartTime().getEpochSecond())
}; };
LOG.debug("Running {}", Arrays.toString(args)); LOG.debug("Running {}", Arrays.toString(args));
Process process = rt.exec(args, OS.getEnvironment()); Process process = rt.exec(args, OS.getEnvironment());
// TODO maybe write these to a separate log file, e.g. recname.ts.pp.log
Thread std = new Thread(new StreamRedirectThread(process.getInputStream(), System.out)); Thread std = new Thread(new StreamRedirectThread(process.getInputStream(), System.out));
std.setName("Process stdout pipe"); std.setName("Process stdout pipe");
std.setDaemon(true); std.setDaemon(true);
@ -246,7 +241,6 @@ public class LocalRecorder implements Recorder {
} catch (Exception e) { } catch (Exception e) {
LOG.error("Error in process thread", e); LOG.error("Error in process thread", e);
} }
}).start();
} }
} }
@ -306,9 +300,9 @@ public class LocalRecorder implements Recorder {
recording = false; recording = false;
LOG.debug("Stopping monitor threads"); LOG.debug("Stopping monitor threads");
processMonitor.running = false; processMonitor.running = false;
postProcessingTrigger.running = false;
LOG.debug("Stopping all recording processes"); LOG.debug("Stopping all recording processes");
stopRecordingProcesses(); stopRecordingProcesses();
ppThreadPool.shutdown();
client.shutdown(); client.shutdown();
} }
@ -318,12 +312,7 @@ public class LocalRecorder implements Recorder {
for (Model model : models) { for (Model model : models) {
Download recordingProcess = recordingProcesses.get(model); Download recordingProcess = recordingProcesses.get(model);
if (recordingProcess != null) { if (recordingProcess != null) {
try { stopRecordingProcess(model);
recordingProcess.stop();
LOG.debug("Stopped recording for {}", model);
} catch (Exception e) {
LOG.error("Couldn't stop recording for model {}", model, e);
}
} }
} }
} finally { } finally {
@ -375,21 +364,14 @@ public class LocalRecorder implements Recorder {
for (Iterator<Entry<Model, Download>> iterator = recordingProcesses.entrySet().iterator(); iterator.hasNext();) { for (Iterator<Entry<Model, Download>> iterator = recordingProcesses.entrySet().iterator(); iterator.hasNext();) {
Entry<Model, Download> entry = iterator.next(); Entry<Model, Download> entry = iterator.next();
Model m = entry.getKey(); Model m = entry.getKey();
Download d = entry.getValue(); Download download = entry.getValue();
if (!d.isAlive()) { if (!download.isAlive()) {
LOG.debug("Recording terminated for model {}", m.getName()); LOG.debug("Recording terminated for model {}", m.getName());
iterator.remove(); iterator.remove();
restart.add(m); restart.add(m);
if(Config.isServerMode()) { fireRecordingStateChanged(download.getTarget(), STOPPED, m, download.getStartTime());
try { Runnable pp = createPostProcessor(download);
finishRecording(d.getTarget()); ppThreadPool.submit(pp);
} catch(Exception e) {
LOG.error("Error while finishing recording for model {}", m.getName(), e);
}
} else {
postprocess(d);
}
fireRecordingStateChanged(d.getTarget(), FINISHED, m, d.getStartTime()); // TODO fire all the events
} }
} }
for (Model m : restart) { for (Model m : restart) {
@ -407,20 +389,6 @@ public class LocalRecorder implements Recorder {
} }
} }
private void finishRecording(File directory) {
if(Config.isServerMode()) {
Thread t = new Thread() {
@Override
public void run() {
generatePlaylist(directory);
}
};
t.setDaemon(true);
t.setName("Post-Processing " + directory.toString());
t.start();
}
}
private void generatePlaylist(File recDir) { private void generatePlaylist(File recDir) {
PlaylistGenerator playlistGenerator = new PlaylistGenerator(); PlaylistGenerator playlistGenerator = new PlaylistGenerator();
playlistGenerators.put(recDir, playlistGenerator); playlistGenerators.put(recDir, playlistGenerator);
@ -445,18 +413,11 @@ public class LocalRecorder implements Recorder {
EventBusHolder.BUS.post(evt); EventBusHolder.BUS.post(evt);
} }
private class PostProcessingTrigger extends Thread { /**
private volatile boolean running = false; * This is called once at start for server mode. When the server is killed, recordings are
* left without playlist. This method creates playlists for them.
public PostProcessingTrigger() { */
setName("PostProcessingTrigger"); private void processUnfinishedRecordings() {
setDaemon(true);
}
@Override
public void run() {
running = true;
while (running) {
try { try {
List<Recording> recs = getRecordings(); List<Recording> recs = getRecordings();
for (Recording rec : recs) { for (Recording rec : recs) {
@ -470,24 +431,14 @@ public class LocalRecorder implements Recorder {
} }
} }
if (!recordingProcessFound) { if (!recordingProcessFound) {
if (deleteInProgress.contains(recDir)) { ppThreadPool.submit(() -> {
LOG.debug("{} is being deleted. Not going to start post-processing", recDir); generatePlaylist(recDir);
} else { });
finishRecording(recDir);
} }
} }
} }
}
if (running)
Thread.sleep(10000);
} catch (InterruptedException e) {
LOG.error("Couldn't sleep", e);
} catch (Exception e) { } catch (Exception e) {
LOG.error("Unexpected error in playlist trigger thread", e); LOG.error("Unexpected error in playlist trigger", e);
}
}
LOG.debug(getName() + " terminated");
} }
} }
@ -781,4 +732,17 @@ public class LocalRecorder implements Recorder {
return getFreeSpaceBytes() > minimum; return getFreeSpaceBytes() > minimum;
} }
} }
private Runnable createPostProcessor(Download download) {
return () -> {
LOG.debug("Starting post-processing for {}", download.getTarget());
if(Config.isServerMode()) {
fireRecordingStateChanged(download.getTarget(), GENERATING_PLAYLIST, download.getModel(), download.getStartTime());
generatePlaylist(download.getTarget());
}
fireRecordingStateChanged(download.getTarget(), POST_PROCESSING, download.getModel(), download.getStartTime());
postprocess(download);
fireRecordingStateChanged(download.getTarget(), FINISHED, download.getModel(), download.getStartTime());
};
}
} }