Fix graceful shutdown

This commit is contained in:
0xb00bface 2023-11-11 11:58:42 +01:00
parent 797c69c06b
commit ef2e354d65
2 changed files with 72 additions and 55 deletions

View File

@ -65,15 +65,16 @@ public class RecordingPreconditions {
} }
private void ensureModelIsOnline(Model model) { private void ensureModelIsOnline(Model model) {
String msg = model.getName() + "'s room is not public";
try { try {
if (!model.isOnline(IGNORE_CACHE)) { if (!model.isOnline(IGNORE_CACHE)) {
throw new PreconditionNotMetException(model.getName() + "'s room is not public"); throw new PreconditionNotMetException(msg);
} }
} catch (IOException | ExecutionException e) { } catch (IOException | ExecutionException e) {
throw new PreconditionNotMetException(model.getName() + "'s room is not public"); throw new PreconditionNotMetException(msg);
} catch (InterruptedException e) { } catch (InterruptedException e) {
Thread.currentThread().interrupt(); Thread.currentThread().interrupt();
throw new PreconditionNotMetException(model.getName() + "'s room is not public"); throw new PreconditionNotMetException(msg);
} }
} }
@ -158,7 +159,7 @@ public class RecordingPreconditions {
} }
private void ensureRecorderIsActive() { private void ensureRecorderIsActive() {
if (!recorder.isRunning()) { if (!recorder.isRunning() || recorder.isShuttingDown()) {
throw new PreconditionNotMetException("Recorder is not in recording mode"); throw new PreconditionNotMetException("Recorder is not in recording mode");
} }
} }
@ -170,7 +171,10 @@ public class RecordingPreconditions {
private void ensureNoOtherFromModelGroupIsRecording(Model model) throws InvalidKeyException, NoSuchAlgorithmException, IOException { private void ensureNoOtherFromModelGroupIsRecording(Model model) throws InvalidKeyException, NoSuchAlgorithmException, IOException {
Optional<ModelGroup> modelGroup = recorder.getModelGroup(model); Optional<ModelGroup> modelGroup = recorder.getModelGroup(model);
if (modelGroup.isPresent()) { if (modelGroup.isEmpty()) {
return;
}
for (String modelUrl : modelGroup.get().getModelUrls()) { for (String modelUrl : modelGroup.get().getModelUrls()) {
if (modelUrl.equals(model.getUrl())) { if (modelUrl.equals(model.getUrl())) {
// no other model with higher prio is online, start recording // no other model with higher prio is online, start recording
@ -191,7 +195,6 @@ public class RecordingPreconditions {
} }
} }
} }
}
private boolean otherModelIsRecorded(Model model) throws InvalidKeyException, NoSuchAlgorithmException, IOException { private boolean otherModelIsRecorded(Model model) throws InvalidKeyException, NoSuchAlgorithmException, IOException {
return recorder.getCurrentlyRecording().contains(model); return recorder.getCurrentlyRecording().contains(model);

View File

@ -40,9 +40,10 @@ import static java.lang.Thread.MIN_PRIORITY;
public class SimplifiedLocalRecorder implements Recorder { public class SimplifiedLocalRecorder implements Recorder {
public static final boolean IGNORE_CACHE = true; public static final boolean IGNORE_CACHE = true;
private List<Model> models = Collections.synchronizedList(new ArrayList<>()); private final List<Model> models = Collections.synchronizedList(new ArrayList<>());
private final Config config; private final Config config;
private volatile boolean running; private volatile boolean running;
private volatile boolean shuttingDown = false;
private final ReentrantLock recorderLock = new ReentrantLock(); private final ReentrantLock recorderLock = new ReentrantLock();
private final ReentrantLock modelGroupLock = new ReentrantLock(); private final ReentrantLock modelGroupLock = new ReentrantLock();
private final RecorderHttpClient client; private final RecorderHttpClient client;
@ -223,7 +224,7 @@ public class SimplifiedLocalRecorder implements Recorder {
} }
private void waitForRecordingsToTerminate() { private void waitForRecordingsToTerminate() {
long secondsToWait = 30; long secondsToWait = TimeUnit.MINUTES.toSeconds(10);
for (int i = 0; i < secondsToWait; i++) { for (int i = 0; i < secondsToWait; i++) {
if (recordingProcesses.isEmpty()) { if (recordingProcesses.isEmpty()) {
return; return;
@ -240,6 +241,26 @@ public class SimplifiedLocalRecorder implements Recorder {
try { try {
postProcessing.submit(() -> { postProcessing.submit(() -> {
try { try {
runPostProcessing(recording);
} catch (Exception e) {
if (e instanceof InterruptedException) { // NOSONAR
Thread.currentThread().interrupt();
}
log.error("Error while post-processing recording {}", recording, e);
recording.setStatus(State.FAILED);
try {
recordingManager.saveRecording(recording);
} catch (IOException e1) {
log.error("Couldn't update recording state for recording {}", recording, e1);
}
}
});
} catch (RejectedExecutionException e) {
log.error("Could not start post-processing for {} {}:{}. Execution rejected by thread pool", recording, recording.getModel().getSite().getName(), recording.getModel().getDisplayName());
}
}
private void runPostProcessing(Recording recording) throws IOException, InterruptedException {
setRecordingStatus(recording, State.POST_PROCESSING); setRecordingStatus(recording, State.POST_PROCESSING);
recording.getRecordingProcess().stop(); recording.getRecordingProcess().stop();
recording.getRecordingProcess().awaitEnd(); recording.getRecordingProcess().awaitEnd();
@ -269,22 +290,6 @@ public class SimplifiedLocalRecorder implements Recorder {
recordingManager.saveRecording(recording); recordingManager.saveRecording(recording);
} }
log.info("Post-processing finished for {}", recording.getModel().getName()); log.info("Post-processing finished for {}", recording.getModel().getName());
} catch (Exception e) {
if (e instanceof InterruptedException) { // NOSONAR
Thread.currentThread().interrupt();
}
log.error("Error while post-processing recording {}", recording, e);
recording.setStatus(State.FAILED);
try {
recordingManager.saveRecording(recording);
} catch (IOException e1) {
log.error("Couldn't update recording state for recording {}", recording, e1);
}
}
});
} catch (RejectedExecutionException e) {
log.error("Could not start post-processing for {} {}:{}. Execution rejected by thread pool", recording, recording.getModel().getSite().getName(), recording.getModel().getDisplayName());
}
} }
private PostProcessingContext createPostProcessingContext(Recording recording) { private PostProcessingContext createPostProcessingContext(Recording recording) {
@ -322,7 +327,7 @@ public class SimplifiedLocalRecorder implements Recorder {
config.getSettings().models.add(Mappers.getMapper(ModelMapper.class).toDto(model)); config.getSettings().models.add(Mappers.getMapper(ModelMapper.class).toDto(model));
config.save(); config.save();
} catch (IOException e) { } catch (IOException e) {
log.error("Couldn't save config", e); errorSavingConfig(e);
} finally { } finally {
recorderLock.unlock(); recorderLock.unlock();
} }
@ -334,6 +339,10 @@ public class SimplifiedLocalRecorder implements Recorder {
} }
} }
private void errorSavingConfig(IOException e) {
log.error("Couldn't save config", e);
}
private void copyModelProperties(Model src, Model existing) { private void copyModelProperties(Model src, Model existing) {
existing.setSuspended(src.isSuspended()); existing.setSuspended(src.isSuspended());
existing.setMarkedForLaterRecording(src.isMarkedForLaterRecording()); existing.setMarkedForLaterRecording(src.isMarkedForLaterRecording());
@ -505,6 +514,7 @@ public class SimplifiedLocalRecorder implements Recorder {
@Override @Override
public void shutdown(boolean immediately) { public void shutdown(boolean immediately) {
log.info("Shutting down"); log.info("Shutting down");
shuttingDown = true;
if (!immediately) { if (!immediately) {
try { try {
stopRecordings(); stopRecordings();
@ -536,7 +546,7 @@ public class SimplifiedLocalRecorder implements Recorder {
getRecordingProcessForModel(model).ifPresent(this::stopRecordingProcess); getRecordingProcessForModel(model).ifPresent(this::stopRecordingProcess);
} catch (IOException e) { } catch (IOException e) {
log.error("Couldn't save config", e); errorSavingConfig(e);
} finally { } finally {
recorderLock.unlock(); recorderLock.unlock();
} }
@ -780,7 +790,7 @@ public class SimplifiedLocalRecorder implements Recorder {
log.warn("Couldn't change priority for model {}. Not found in list", model.getName()); log.warn("Couldn't change priority for model {}. Not found in list", model.getName());
} }
} catch (IOException e) { } catch (IOException e) {
log.error("Couldn't save config", e); errorSavingConfig(e);
} finally { } finally {
recorderLock.unlock(); recorderLock.unlock();
} }
@ -828,6 +838,10 @@ public class SimplifiedLocalRecorder implements Recorder {
return running; return running;
} }
boolean isShuttingDown() {
return shuttingDown;
}
List<Recording> getRecordingProcesses() { List<Recording> getRecordingProcesses() {
return recordingProcesses; return recordingProcesses;
} }